# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import re
import os
import csv
import pandas as pd
from utils.HttpUtils import HttpUtils


requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

"""
新浪微博-搜索关键词
"""


class Test_Get():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        # requests.utils.dict_from_cookiejar(html.cookies)  # 获取cookies转dict
        self.cookie = cookiejar.CookieJar()
        # ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            # "User-Agent": ua.random
            # 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15'
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Connection': 'keep-alive',
            'Cookie': 'login_sid_t=8ae7de674ed2a79b59dfa5f2cba33384; cross_origin_proto=SSL; _s_tentry=passport.weibo.com; Apache=9415485890987.03.1605788075159; SINAGLOBAL=9415485890987.03.1605788075159; ULV=1605788075167:1:1:1:9415485890987.03.1605788075159:; ALF=1637324029; SSOLoginState=1605788029; SUB=_2A25yshEuDeRhGeNN41sX8CjPyjWIHXVRxgXmrDV8PUNbmtAKLWbakW9NSZ0OGQrS2dM3Ke4razTrgUewX2LPGXgH; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWyIgAfvDAPGCkEq6N8Kf4q5JpX5KzhUgL.Fo-01h.cehq0eK.2dJLoI0.LxK-LBo5L12qLxKML12eLB-2LxK-L1KeLB.eLxK-L1KeLB.eLxK-L1KBLBKMLxK-L1KBLBKMXeoqt; wvr=6; webim_unReadCount=%7B%22time%22%3A1605788137912%2C%22dm_pub_total%22%3A0%2C%22chat_group_client%22%3A0%2C%22chat_group_notice%22%3A0%2C%22allcountNum%22%3A7%2C%22msgbox%22%3A0%7D',
            'DNT': '1',
            'Host': 's.weibo.com',
            'Referer': 'https://s.weibo.com/hot?q=%23%E5%85%AC%E7%9B%8A%23&xsort=hot&suball=1&tw=hotweibo&Refer=weibo_hot',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'same-origin',
            'Sec-Fetch-User': '?1',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'
        }

    # 获取微博内容
    def get_contents_pc(self, cursor):
        postData = {
            'q': '#公益#',
            'xsort': 'hot',
            'suball': '1',
            'tw': 'hotweibo',
            'Refer': 'weibo_hot',
            'page': cursor
        }
        # 话题标志。如果是话题，跳过第一条
        ht_flag = True
        url = "https://s.weibo.com/weibo"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        root = etree.HTML(html.text)
        # for r in root.xpath('//div[@class="card-wrap"]'):
        for r in root.xpath('//div[@class="card-wrap" and @action-type="feed_list_item"]'):
            # 如果是话题，则跳过第一条
            if ht_flag == True:
                dict_data = dict()
                mid = r.xpath('./@mid')[0]  # 评论ID
                dict_data['用户名'] = r.xpath('.//div[@class="info"]/div/a[1]/text()')[0]  # 微博名
                dict_data['认证用户'] = "".join(r.xpath('.//div[@class="info"]/div/a[2]/@title'))  # 认证
                if len(r.xpath('.//div[@class="content"]/p[@node-type="feed_list_content_full"]')) > 0:
                    dict_data['内容'] = "".join(r.xpath('.//div[@class="content"]/p[@node-type="feed_list_content_full"]//text()')).replace("\n", "").strip()  # 内容
                else:
                    dict_data['内容'] = "".join(r.xpath('.//div[@class="content"]/p[@node-type="feed_list_content"]//text()')).replace("\n", "").strip()  # 内容
                dict_data['发布时间'] = r.xpath('.//p[@class="from"]/a[1]/text()')[0].replace("\n", "").strip()  # 发布时间
                dict_data['终端'] = "".join(r.xpath('.//p[@class="from"]/a[2]/text()'))  # 终端
                dict_data['收藏'] = r.xpath('.//div[@class="card-act"]/ul/li[1]/a//text()')[0]  # 收藏
                dict_data['转发'] = r.xpath('.//div[@class="card-act"]/ul/li[2]/a//text()')[0]  # 转发
                dict_data['评论'] = r.xpath('.//div[@class="card-act"]/ul/li[3]/a//text()')[0]  # 评论
                dict_data['赞'] = "".join(r.xpath('.//div[@class="card-act"]/ul/li[4]/a/em/text()'))  # 赞
                dict_data['转发用户'] = ""
                dict_data['转发内容'] = ""
                dict_data['转发时间'] = ""
                dict_data['转发终端'] = ""
                # 转发微博
                if len(r.xpath('.//div[@class="card-comment"]')) > 0:
                    dict_data['转发用户'] = r.xpath('.//div[@node-type="feed_list_forwardContent"]/a/text()')[0]  # 原创作者
                    if len(r.xpath('.//div[@node-type="feed_list_forwardContent"]/p[@node-type="feed_list_content_full"]')) > 0:
                        dict_data['转发内容'] = "".join(r.xpath('.//div[@node-type="feed_list_forwardContent"]/p[@node-type="feed_list_content_full"]//text()')).replace("\n", "").strip()  # 内容
                    else:
                        dict_data['转发内容'] = "".join(r.xpath('.//div[@node-type="feed_list_forwardContent"]/p[@node-type="feed_list_content"]//text()')).replace("\n", "").strip()  # 内容
                    dict_data['转发时间'] = r.xpath('.//div[@class="func"]/p/a[1]/text()')[0]  # 原创作者
                    dict_data['转发终端'] = r.xpath('.//div[@class="func"]/p/a[2]/text()')[0]  # 原创作者

                print(str(dict_data))
                self.dictToCsv(dict_data, "aa")
                # # 获取评论内容
                # self.get_comments(mid, "0")
            ht_flag = True


    # 获取评论内容
    def get_comments(self, mid, max_id):
        headers_comments = {
            "cookie": "_T_WM=87800207856; WEIBOCN_FROM=1110006030; ALF=1585493138; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWyIgAfvDAPGCkEq6N8Kf4q5JpX5K-hUgL.Fo-01h.cehq0eK.2dJLoI0.LxK-LBo5L12qLxKML12eLB-2LxK-L1KeLB.eLxK-L1KeLB.eLxK-L1KBLBKMLxK-L1KBLBKMXeoqt; MLOGIN=1; SCF=AneWmT9x5LGhR8-HnrzdQ1GR01azLi-pHCernHokw_RTW0BCwlZPENVg8_fqS5j1xcsXxROwVgrJNiwQPWxcB9k.; SUB=_2A25zXVfLDeThGeNN41sX8CjPyjWIHXVQvnmDrDV6PUJbktANLWfekW1NSZ0OGUYGcrDAtEcyzvsmAILyAwVg3oup; SUHB=0y0Jj1Yhg9zs2E; SSOLoginState=1582901147; XSRF-TOKEN=4df66a; M_WEIBOCN_PARAMS=oid%3D4476674161119611%26luicode%3D10000011%26lfid%3D1005051669879400%26uicode%3D20000061%26fid%3D4476674161119611",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36",
        }
        postData = {
            "id": mid,  # "4456240593602073",  # 评论ID
            "mid": mid,  # "4456240593602073",
            "max_id_type": "0",
        }
        # 第一次无max_id参数 游标
        if max_id != "0":
            postData['max_id'] = str(max_id)
        url = "https://m.weibo.cn/comments/hotflow"
        html_comments = HttpUtils.do_request("GET", url, headers_comments, postData)
        data_json = json.loads(html_comments.text)
        num = 0  # 计算已获取评论数量
        if data_json['ok'] == 1:
            count = data_json['data']['total_number']  # 总数量
            max_id = data_json['data']['max_id']  # 下页游标
            for d in data_json['data']['data']:
                num += 1
                dict_data = dict()
                dict_data['id'] = d['id']  # 评论ID
                # 评论时间
                timeStruct = time.strptime(d['created_at'], '%a %b %d %X %z %Y')
                dict_data['created_at'] = time.strftime('%Y-%m-%d %H:%M:%S', timeStruct)
                dict_data['text'] = d['text']  # 评论内容
                dict_data['total_number'] = d['total_number']  # 回复数量
                dict_data['like_count'] = d['like_count']  # 点赞数量
                dict_data['user_id'] = d['user']['id']  # 用户id
                dict_data['user_name'] = d['user']['screen_name']  # 用户名称
                print("---------------------")
                print(str(dict_data))

                # 限定获取评论数量
                # if num >= 10:
                #     return
            self.get_comments(id, max_id)
        else:
            print("==========获取评论结束==================")


if __name__ == '__main__':
    test_get = Test_Get()
    for i in range(1, 100):
        print(f"======={i}==========")
        test_get.get_contents_pc(i)
